//============================================================================
// Name        : ParserRefinado.cpp
//============================================================================

#include <iostream>
#include <fstream>
#include <string>
#include <list>

#include <curl/curl.h>
#include <curl/types.h>
#include <curl/easy.h>

#include "URL.h"
#include "URLNormalizer.h"
#include "DebugLogger.h"

using namespace std;


struct data { char trace_ascii; /* 1 or 0 */ };
string data_site;
string header_site;
string site_url;
int res_code;
URLNormalizer normalisa;
//std::list<URL> buffer_urls;
std::string buffer_page;
std::string url_rejeita;
std::string pagina_at;
std::string s_href = "href";
std::list<std::string> buffer_links;	


CURL *curl;


list<URL> eliminaManjados(list<URL>);


size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *userp){
  size_t realsize = size * nmemb;
  char *teste = (char *) ptr;
  string *data_str = (string *) userp;
  for(uint i=0; i < nmemb; i++) *data_str += teste[i];
  return realsize;
}


int curl_config(void) {
	 CURLcode res;
	 struct data config;
	 config.trace_ascii = 1; /* enable ascii tracing */
	 if(curl) {
       curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
       curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data_site);
       curl_easy_setopt(curl, CURLOPT_VERBOSE, 0);
	   //Tamanho maximo para arquivo (em bytes)
	   curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, 300*1024); //limita 300 kbytes
	   //Limite de velocidade (em bytes por segundo)
       //    CURLOPT_LOW_SPEED_LIMIT
	   //redirecionamentos
	   curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
//	   curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3); //maximo Redirecionamento. se Follow true            
	   curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5); //sem efeito, se NOSIGNAL is true
    //    deve setar para TRUE se em multi-Thread
	   curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
	//    curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
	//    curl_easy_setopt(curl, CURLOPT_HEADER, 1);
	//	  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
	//    curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
	//    http://www.pudim.com.br/robots.txt
	//   --> MC: entre 5 e 6 milhao
	//   --> Baggio: entre 1 e 2 milhaodata_site
	//    http://www.ogrilofeliz.com.br/robots.txt
	   res = curl_easy_perform(curl);
	   res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &res_code);    
	//    cout << curl_easy_strerror(res);  // imprime erro bonitinho em char*
	 }
	 return 0;
}

string getPage(){
		curl = curl_easy_init();  
		if(curl){
			curl_easy_setopt(curl, CURLOPT_URL, site_url.c_str());
			curl_config();
		}
//		cout << data_site << endl;
		curl_easy_cleanup(curl);
		if(res_code == 200) return data_site;
		else                return "";
}


string achaDominio (string str) {
	uint index = str.find("://");
	if (index != string::npos) { 
		str = str.substr( index+3, str.size() );
	}
	uint slashIndex = str.find("/");
	if (slashIndex != string::npos) 
		return str.substr(0, slashIndex);
	return str;
}

void rejeita_links(std::list <std::string> &lista_in){
	DebugLogger dbug_rejeita_links("dbug_rejeita_links");
	std::string temp_url;
	int buffer_links_size = (int) lista_in.size();
	try{		
		url_rejeita = "()'!@{}|\\^[]`<>";
		//referencia para links relativos
		uint pos_barra = pagina_at.find_last_of("/"); 
		if( pos_barra != std::string::npos) pagina_at.erase(pos_barra);
		//Grava em arquivo a lista
		for( int i=0; i<buffer_links_size; i++){
			temp_url = lista_in.front();
			if( temp_url.find_first_of(url_rejeita) == std::string::npos
				&& temp_url.find_first_of("./") != std::string::npos ){
					// Tira carecteres indesejados
					for(unsigned int t=0; t < temp_url.size(); t++)
						if(temp_url.at(t) == ' ' || temp_url.at(t) == '\n'){
							temp_url.erase(t,1); // tira caractere
							t--;           // volta uma pos.
						}
					//resolve links relativos
					if (temp_url.at(0) == '/') {
						temp_url = achaDominio(pagina_at) + temp_url;
					}
					
					if(temp_url.at(0) == '.'){
						for(uint t=0; t<=temp_url.size(); t++)
							if(temp_url.at(0) == '/' || temp_url.at(0) == '.'){
								if(temp_url.at(0) == '.' && temp_url.at(1) == '.'){
									temp_url.erase(0,2);
									pos_barra = pagina_at.find_last_of("/"); 
									if( pos_barra != std::string::npos){
										if( pagina_at.find_last_of("/") > pagina_at.find_last_of(".")  
											&& pagina_at.find_last_of("/") != std::string::npos
											&& pagina_at.find_last_of("/") != std::string::npos) pagina_at.erase(pos_barra); //retorna dir se puder
									}
								}
								temp_url.erase(0,1);
								if(temp_url.find_first_of("./") == std::string::npos){
									temp_url = pagina_at + '/' + temp_url; 
									break;
								}
							}else{
								temp_url = pagina_at + '/' + temp_url;
								break;
							}
					}else if(   temp_url.find("://") == std::string::npos &&
							  ( temp_url.find_first_of("/") < temp_url.find_first_of(".") ||
							    temp_url.find_first_of("/") == std::string::npos )){
						temp_url = pagina_at + '/' + temp_url;
					}
					lista_in.push_back(temp_url);
					lista_in.pop_front();
			}
			
			else{
				lista_in.pop_front();
			}
		}
	}catch(...){ // Gambi
		dbug_rejeita_links.enable(DebugLogger::info + DebugLogger::alert + DebugLogger::error);
		dbug_rejeita_links.errorMsg(lista_in.front());
	}
}



list<URL>  parser(string buffer_page){
	unsigned int count_link = 0;	
	list<string> buffer_links;
	list<URL> buffer_urls;
	
	std::string::size_type pos_final;
	std::string::size_type pos_atual;
	
	//acha posicao final do arquivo	url_rejeita = "()'!";
	pos_final = buffer_page.size();
	pos_atual = buffer_page.find(s_href, 0);
	do{
		std::string::size_type pos_aspa_begin;
		std::string::size_type pos_aspa_end;
		if(pos_atual >= pos_final) continue;
		//pega o link, se existe
		pos_aspa_begin = buffer_page.find("\"", pos_atual);
		pos_aspa_end = buffer_page.find("\"",pos_aspa_begin+1);
		buffer_links.push_back(buffer_page.substr(pos_aspa_begin+1, pos_aspa_end-pos_aspa_begin-1));
		pos_atual = buffer_page.find(s_href, pos_atual+1);
		count_link++;
	}while(pos_atual < pos_final);
	
	// Aplica regras para extrair falsos links
	rejeita_links(buffer_links);
	// Normalize
	buffer_urls = eliminaManjados( normalisa.normalize(buffer_links) );  // canoniza URLs
	
	return buffer_urls;
}


list<URL> eliminaManjados(list<URL> linkList) {
	list<string> listaDeFormatosManjados;
	list<URL> listaFiltrada;
	listaDeFormatosManjados.push_front(".jpg");
	listaDeFormatosManjados.push_front(".pdf");
	listaDeFormatosManjados.push_front(".doc");
	listaDeFormatosManjados.push_front(".pps");
	listaDeFormatosManjados.push_front(".gif");
	listaDeFormatosManjados.push_front(".bmp");
	listaDeFormatosManjados.push_front(".swf");
	listaDeFormatosManjados.push_front(".mp3");
	listaDeFormatosManjados.push_front(".avi");
	listaDeFormatosManjados.push_front(".ico");
	listaDeFormatosManjados.push_front(".mpeg");
	listaDeFormatosManjados.push_front(".mpg");
	listaDeFormatosManjados.push_front(".jpeg");
	listaDeFormatosManjados.push_front(".png");
	listaDeFormatosManjados.push_front(".cdr");
	listaDeFormatosManjados.push_front(".wav");
	
	list<string>::iterator stringIt;
	list<URL>::iterator urlIt;
	
	string path;
	for (urlIt=linkList.begin(); urlIt!=linkList.end(); urlIt++) {
		listaFiltrada.push_front(*urlIt);
		for (stringIt=listaDeFormatosManjados.begin(); stringIt!=listaDeFormatosManjados.end(); stringIt++) {
			path = urlIt->get_path();
			if ( path.rfind(*stringIt) == path.size() - stringIt->size() ) {
				listaFiltrada.pop_front();
				break;
			}
		}
	}
	
	return listaFiltrada;
	
}








// Referencia: http://www.ietf.org/rfc/rfc2396.txt
//             http://gbiv.com/protocols/uri/rfc/rfc3986.html
//   reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
//   unreserved  = alphanum | mark
//   mark        = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
//  Excluidos:
//  control     = <US-ASCII coded characters 00-1F and 7F hexadecimal>
//  space       = <US-ASCII coded character 20 hexadecimal>
//  delims      = "<" | ">" | "#" | "%" | <">
//  unwise      = "{" | "}" | "|" | "\" | "^" | "[" | "]" | "`""
		




int main() {
	string pagina;
	list<URL> links;
	
	ofstream pagina_out("pagina.txt");
	ofstream links_out("links.txt");
	
	//carrega site
	site_url = "www.globo.com/";
//	site_url = "www.ita.br/";
//	site_url = "http://www.dpreview.com/news/0009/00090101canoneosd30samples.asp";
	
	
	pagina_at = site_url;
	pagina = getPage();
	links = parser(pagina);
	
	//Coloca HTML no arquivo
	pagina_out << data_site;
	
	//Coloca links no arquivo
	list<URL>::iterator it;
	for ( it=links.begin() ; it != links.end(); it++ ) links_out << it->toString() << endl;
	
	cout << "Parser Refinado..." << endl;
	cout << "Code: " << res_code << endl;
	cout << "Tamanho:  " << links.size() << endl;
	return 0;
}
